import torch
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime
import io
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"{device}" " is available.")

cpu is available.

from deep_xf.main import *
from deep_xf.dpp import *
from deep_xf.forecast_ml import *
from deep_xf.forecast_ml_extension import *
from deep_xf.stats import *
from deep_xf.utility import *
from deep_xf.denoise import *
from deep_xf.similarity import *

Imported DeepXF version:0.0.1. Example call by using:

***********************   SET MODEL/BASE CONFIGURATIONS   ************************

select_model, select_user_path, select_scaler, forecast_window = Forecast.set_model_config(select_model='rnn', select_user_path='./forecast_folder_path/', select_scaler='minmax', forecast_window=1)

----------------------------------------------------------------------------------

model_df, orig_df = Helper.get_variable(df, ts, fc)

----------------------------------------------------------------------------------

hidden_dim, layer_dim, batch_size, dropout, n_epochs, learning_rate, weight_decay = Forecast.hyperparameter_config(hidden_dim=64,layer_dim = 3, batch_size=64, dropout = 0.2, n_epochs = 30, learning_rate = 1e-3, weight_decay = 1e-6)

**********************   DEEP LEARNING BASED FORECASTING   ***********************

opt, scaler = Forecast.train(df=df_full_features, target_col='value', split_ratio=0.2, select_model=select_model, select_scaler=select_scaler, forecast_window=forecast_window, batch_size=batch_size, hidden_dim=hidden_dim, layer_dim=layer_dim,dropout=dropout, n_epochs=n_epochs, learning_rate=learning_rate, weight_decay=weight_decay)

----------------------------------------------------------------------------------

forecasted_data, ff_full_features, ff_full_features_ = Forecast.forecast(model_df, ts, fc, opt, scaler, period=25, fq='h', select_scaler=select_scaler,)

---------------------------------------------------------------------------------

Helper.explainable_forecast(df_full_features, ff_full_features_, fc, specific_prediction_sample_to_explain=145370, input_label_index_value=0, num_labels=1)


******************   DYNAMIC FACTOR MODEL BASED NOWCASTING   *********************

select_model, select_user_path, select_scaler, forecast_window = Forecast.set_model_config(select_model='em', select_user_path='./forecast_folder_path/', select_scaler='minmax', forecast_window=5)

----------------------------------------------------------------------------------

nowcast_full_data, nowcast_pred_data = EMModel.nowcast(df_full_features, ts, fc, period=5, fq='h', forecast_window=forecast_window, select_model=select_model)

----------------------------------------------------------------------------------

EMModel.explainable_nowcast(df_full_features, nowcast_pred_data, fc, specific_prediction_sample_to_explain=145370, input_label_index_value=0, num_labels=1)

df = pd.read_csv('../data/PJME_hourly.csv')

df.shape

(145366, 2)

df.columns

Index(['Datetime', 'PJME_MW'], dtype='object')

df.head(2)

	Datetime	PJME_MW
0	2002-12-31 01:00:00	26498.0
1	2002-12-31 02:00:00	25147.0

df.tail(2)

	Datetime	PJME_MW
145364	2018-01-01 23:00:00	40164.0
145365	2018-01-02 00:00:00	38608.0

df

	Datetime	PJME_MW
0	2002-12-31 01:00:00	26498.0
1	2002-12-31 02:00:00	25147.0
2	2002-12-31 03:00:00	24574.0
3	2002-12-31 04:00:00	24393.0
4	2002-12-31 05:00:00	24860.0
...	...	...
145361	2018-01-01 20:00:00	44284.0
145362	2018-01-01 21:00:00	43751.0
145363	2018-01-01 22:00:00	42402.0
145364	2018-01-01 23:00:00	40164.0
145365	2018-01-02 00:00:00	38608.0

145366 rows × 2 columns

# set variables
ts, fc = Forecast.set_variable(ts='Datetime', fc='PJME_MW')

ts = Datetime
fc = PJME_MW

# get variables
model_df, orig_df = Helper.get_variable(df, ts, fc)

model_df.shape

(145366, 1)

model_df.head(1)

	value
Datetime
2002-01-01 01:00:00	30393.0

orig_df.shape

(145366, 2)

orig_df.head(1)

	Datetime	PJME_MW
0	2002-12-31 01:00:00	26498.0

EDA

ExploratoryDataAnalysis.plot_dataset(df=model_df,title='PJM East (PJME) Region: estimated energy consumption in Megawatts (MW)')

Feature Engg

df_full_features = Features.generate_date_time_features_hour(model_df, ['hour','month','day','day_of_week','week_of_year'])

df_full_features.columns

Index(['value', 'hour', 'day', 'month', 'day_of_week', 'week_of_year'], dtype='object')

# generating cyclic features
df_full_features = Features.generate_cyclic_features(df_full_features, 'hour', 24, 0)
df_full_features = Features.generate_cyclic_features(df_full_features, 'day_of_week', 7, 0)
df_full_features = Features.generate_cyclic_features(df_full_features, 'month', 12, 1)
df_full_features = Features.generate_cyclic_features(df_full_features, 'week_of_year', 52, 0)

df_full_features.head(1)

	value	day	sin_hour	cos_hour	sin_day_of_week	cos_day_of_week	sin_month	cos_month	sin_week_of_year	cos_week_of_year
Datetime
2002-01-01 01:00:00	30393.0	1	0.258819	0.965926	0.781831	0.62349	0.0	1.0	0.120537	0.992709

df_full_features = Features.generate_other_related_features(df=df_full_features)

df_full_features.head(1)

	value	day	sin_hour	cos_hour	sin_day_of_week	cos_day_of_week	sin_month	cos_month	sin_week_of_year	cos_week_of_year	is_holiday
Datetime
2002-01-01 01:00:00	30393.0	1	0.258819	0.965926	0.781831	0.62349	0.0	1.0	0.120537	0.992709	1

df_full_features.shape

(145366, 11)

select_model, select_user_path, select_scaler, forecast_window = Forecast.set_model_config(select_model='rnn', select_user_path='./forecast_MY/', select_scaler='minmax', forecast_window=1)

select_model = rnn
select_user_path = ./forecast_MY/
select_scaler = minmax
forecast_window = 1

hidden_dim, layer_dim, batch_size, dropout, n_epochs, learning_rate, weight_decay = \
Forecast.hyperparameter_config(hidden_dim=64, layer_dim = 3, batch_size=64, dropout = 0.2, \
                               n_epochs = 30, learning_rate = 1e-3, weight_decay = 1e-6)

hidden_dim = 64
layer_dim = 3
batch_size = 64
dropout = 0.2
n_epochs = 30
learning_rate = 0.001
weight_decay = 1e-06

opt, scaler = Forecast.train(df=df_full_features, target_col='value', split_ratio=0.2, select_model=select_model, \
             select_scaler=select_scaler, forecast_window=forecast_window, batch_size=batch_size,\
            hidden_dim=hidden_dim, layer_dim=layer_dim,dropout=dropout, \
             n_epochs=n_epochs, learning_rate=learning_rate, weight_decay=weight_decay)

Train-Val-Test Split
Predictors: Train-(87219, 10), Val-(29073, 10), Test-(29074, 10)
Response: Train-(87219, 1), Val-(29073, 1), Test-(29074, 1)
cpu is available.
[1/30] Training loss: 0.0116	 Validation loss: 0.0162
[2/30] Training loss: 0.0112	 Validation loss: 0.0166
[3/30] Training loss: 0.0116	 Validation loss: 0.0165
[4/30] Training loss: 0.0122	 Validation loss: 0.0162
[5/30] Training loss: 0.0122	 Validation loss: 0.0162
[6/30] Training loss: 0.0122	 Validation loss: 0.0162
[7/30] Training loss: 0.0123	 Validation loss: 0.0162
[8/30] Training loss: 0.0122	 Validation loss: 0.0160
[9/30] Training loss: 0.0125	 Validation loss: 0.0162
[10/30] Training loss: 0.0123	 Validation loss: 0.0165
[11/30] Training loss: 0.0118	 Validation loss: 0.0167
[12/30] Training loss: 0.0113	 Validation loss: 0.0176
[13/30] Training loss: 0.0115	 Validation loss: 0.0145
[14/30] Training loss: 0.0099	 Validation loss: 0.0106
[15/30] Training loss: 0.0093	 Validation loss: 0.0114
[16/30] Training loss: 0.0091	 Validation loss: 0.0117
[17/30] Training loss: 0.0090	 Validation loss: 0.0128
[18/30] Training loss: 0.0089	 Validation loss: 0.0125
[19/30] Training loss: 0.0088	 Validation loss: 0.0134
[20/30] Training loss: 0.0086	 Validation loss: 0.0115
[21/30] Training loss: 0.0085	 Validation loss: 0.0129
[22/30] Training loss: 0.0086	 Validation loss: 0.0123
[23/30] Training loss: 0.0086	 Validation loss: 0.0121
[24/30] Training loss: 0.0086	 Validation loss: 0.0130
[25/30] Training loss: 0.0086	 Validation loss: 0.0122
[26/30] Training loss: 0.0083	 Validation loss: 0.0103
[27/30] Training loss: 0.0082	 Validation loss: 0.0104
[28/30] Training loss: 0.0081	 Validation loss: 0.0099
[29/30] Training loss: 0.0080	 Validation loss: 0.0106
[30/30] Training loss: 0.0079	 Validation loss: 0.0106

Forecast testset predictions:                        value    prediction
Datetime                                  
2015-04-09 15:00:00  32204.0  32155.353516
2015-04-09 16:00:00  32049.0  32166.613281
2015-04-09 17:00:00  32209.0  32159.548828
2015-04-09 18:00:00  32707.0  32128.466797
2015-04-09 19:00:00  33012.0  32061.867188
...                      ...           ...
2018-08-02 20:00:00  44057.0  42633.503906
2018-08-02 21:00:00  43256.0  41089.332031
2018-08-02 22:00:00  41552.0  37979.539062
2018-08-02 23:00:00  38500.0  33391.750000
2018-08-03 00:00:00  35486.0  29462.552734

[29074 rows x 2 columns]
Mean Absolute Error (MAE):        3175.7388
Root Mean Squared Error (RMSE):    4138.234164471604
Forecast bias:   		   545.1050079108481
Mean Absolute Percentage Error (MAPE):    9.965622
R^2 Score:                  0.5932845083635333
Explained Variance Score:   0.6003414988517761
Root Mean Squared Relative Error (RMSRE):  0.122882746
Symmetric Mean Absolute Percentage Error (sMAPE):    0.10098717361688614
pcc coefficient:  0.7749467302680185
Model Evaluations: {'mae': 3175.7388, 'rmse': 4138.234164471604, 'fc_bias': 545.1050079108481, 'mape': 9.965622}
Model Evaluations: {'r2': 0.5932845083635333, 'evs': 0.6003414988517761, 'rmsre': 0.122882746, 'smape': 0.10098717361688614, 'r': 0.7749467302680185}
Mean Absolute Error (MAE):        4487.783057395446
Root Mean Squared Error (RMSE):    5441.424274307522
Forecast bias:   		   -1301.921567344853
Mean Absolute Percentage Error (MAPE):    14.95819082699982
R^2 Score:                  0.2967887938366345
Explained Variance Score:   0.3370447592702236
Root Mean Squared Relative Error (RMSRE):  0.18165206970633552
Symmetric Mean Absolute Percentage Error (sMAPE):    0.14184090730169824
pcc coefficient:  0.5810578240879188

forecasted_data, ff_full_features, ff_full_features_ = Forecast.forecast(model_df, ts, fc, opt, scaler, period=25, fq='h', select_scaler=select_scaler,)

Forecast period dataframe: DatetimeIndex(['2018-08-03 01:00:00', '2018-08-03 02:00:00',
               '2018-08-03 03:00:00', '2018-08-03 04:00:00',
               '2018-08-03 05:00:00', '2018-08-03 06:00:00',
               '2018-08-03 07:00:00', '2018-08-03 08:00:00',
               '2018-08-03 09:00:00', '2018-08-03 10:00:00',
               '2018-08-03 11:00:00', '2018-08-03 12:00:00',
               '2018-08-03 13:00:00', '2018-08-03 14:00:00',
               '2018-08-03 15:00:00', '2018-08-03 16:00:00',
               '2018-08-03 17:00:00', '2018-08-03 18:00:00',
               '2018-08-03 19:00:00', '2018-08-03 20:00:00',
               '2018-08-03 21:00:00', '2018-08-03 22:00:00',
               '2018-08-03 23:00:00', '2018-08-04 00:00:00',
               '2018-08-04 01:00:00'],
              dtype='datetime64[ns]', name='Datetime', freq=None)
Tensor size: 25
Forecast period predictions:                             value
Datetime                         
2018-08-03 01:00:00  34638.609375
2018-08-03 02:00:00  33303.523438
2018-08-03 03:00:00  32243.546875
2018-08-03 04:00:00  31605.658203
2018-08-03 05:00:00  31528.224609
2018-08-03 06:00:00  32164.652344
2018-08-03 07:00:00  33943.816406
2018-08-03 08:00:00  37538.597656
2018-08-03 09:00:00  41112.054688
2018-08-03 10:00:00  42940.699219
2018-08-03 11:00:00  43869.062500
2018-08-03 12:00:00  44411.867188
2018-08-03 13:00:00  44743.769531
2018-08-03 14:00:00  44945.308594
2018-08-03 15:00:00  45061.054688
2018-08-03 16:00:00  45115.039062
2018-08-03 17:00:00  45117.359375
2018-08-03 18:00:00  45065.292969
2018-08-03 19:00:00  44933.492188
2018-08-03 20:00:00  44634.777344
2018-08-03 21:00:00  43916.835938
2018-08-03 22:00:00  42173.613281
2018-08-03 23:00:00  39058.500000
2018-08-04 00:00:00  35432.781250
2018-08-04 01:00:00  34132.726562

Helper.explainable_forecast(df_full_features, ff_full_features_, fc, specific_prediction_sample_to_explain=df_full_features.shape[0]+2, input_label_index_value=0, num_labels=1)

Epoch 1/30
4543/4543 [==============================] - 2s 439us/step - loss: 25422.3906
Epoch 2/30
4543/4543 [==============================] - 2s 436us/step - loss: 13379.1924
Epoch 3/30
4543/4543 [==============================] - 2s 446us/step - loss: 12765.8008
Epoch 4/30
4543/4543 [==============================] - 2s 434us/step - loss: 12206.8672
Epoch 5/30
4543/4543 [==============================] - 2s 429us/step - loss: 11499.8799
Epoch 6/30
4543/4543 [==============================] - 2s 431us/step - loss: 10565.8955
Epoch 7/30
4543/4543 [==============================] - 2s 430us/step - loss: 9337.6758
Epoch 8/30
4543/4543 [==============================] - 2s 452us/step - loss: 7793.0386
Epoch 9/30
4543/4543 [==============================] - 2s 437us/step - loss: 6072.8354
Epoch 10/30
4543/4543 [==============================] - 2s 430us/step - loss: 4686.6343
Epoch 11/30
4543/4543 [==============================] - 2s 428us/step - loss: 4210.5171
Epoch 12/30
4543/4543 [==============================] - 2s 426us/step - loss: 4133.1021
Epoch 13/30
4543/4543 [==============================] - 2s 426us/step - loss: 4108.6475
Epoch 14/30
4543/4543 [==============================] - 2s 441us/step - loss: 4095.4810
Epoch 15/30
4543/4543 [==============================] - 2s 462us/step - loss: 4088.1372
Epoch 16/30
4543/4543 [==============================] - 2s 438us/step - loss: 4083.3862
Epoch 17/30
4543/4543 [==============================] - 2s 433us/step - loss: 4078.3030
Epoch 18/30
4543/4543 [==============================] - 2s 433us/step - loss: 4072.4236
Epoch 19/30
4543/4543 [==============================] - 2s 470us/step - loss: 4064.1030
Epoch 20/30
4543/4543 [==============================] - 2s 447us/step - loss: 4051.3462
Epoch 21/30
4543/4543 [==============================] - 2s 474us/step - loss: 4032.6768
Epoch 22/30
4543/4543 [==============================] - 2s 430us/step - loss: 4006.9294
Epoch 23/30
4543/4543 [==============================] - 2s 429us/step - loss: 3975.2256
Epoch 24/30
4543/4543 [==============================] - 2s 440us/step - loss: 3940.2900
Epoch 25/30
4543/4543 [==============================] - 2s 429us/step - loss: 3903.0371
Epoch 26/30
4543/4543 [==============================] - 2s 431us/step - loss: 3864.3789
Epoch 27/30
4543/4543 [==============================] - 2s 431us/step - loss: 3824.5417
Epoch 28/30
4543/4543 [==============================] - 2s 428us/step - loss: 3784.0063
Epoch 29/30
4543/4543 [==============================] - 2s 440us/step - loss: 3741.9946
Epoch 30/30
4543/4543 [==============================] - 2s 456us/step - loss: 3699.4485
4543/4543 [==============================] - 2s 356us/step - loss: 3678.8657
[[28038.465]]
day                 3.000000
sin_hour            0.707107
cos_hour            0.707107
sin_day_of_week    -0.433884
cos_day_of_week    -0.900969
sin_month          -0.500000
cos_month          -0.866025
sin_week_of_year   -0.568065
cos_week_of_year   -0.822984
is_holiday          0.000000
Name: 2018-08-03 03:00:00, dtype: float64
Dropdown(description='Select Label:', options=(('PJME_MW_0', 0),), value=0)
Current label Shown: PJME_MW_0

Current Label Shown: PJME_MW_0

Current Label Shown: PJME_MW_0